import json, gzip
from collections import Counter

orfile = gzip.open("split_dictionaries.json.gz",'r')

dictionary = Counter()

iterator = 0
empty = 0
buf = "start"
print "Reading dictionaries"
while buf != "":
	buf = orfile.readline()
	try:
		if not buf.startswith("{}"):
			iterator += 1
			dictionary.update(Counter(json.loads(buf)))
			print "Dictionaries added: ", iterator
		else: empty += 1
	except:
		pass

orfile.close()

print "Writing complete dictionaries to file"
orfile = gzip.open("collection_word_count.json.gz",'w')
orfile.write(json.dumps(dictionary)+"\n")
orfile.close()
print "Done"
print "Getting all words in alphabetical order"
sorted_dictionary = sorted(dictionary.keys())
words_in_collection = sum(dictionary.values())
print "Total words in collection: ", words_in_collection

print "Writing all words in collection to file"
orfile = gzip.open("words_in_document_collection.json.gz",'w')
orfile.write(json.dumps(sorted_dictionary)+"\n")
orfile.close()

print "Calculating frequencies"
frequencies = Counter()

iterator = 0
total = len(sorted_dictionary)
for token in sorted_dictionary:
	iterator += 1
	frequencies[token] = words_in_collection / float(dictionary[token])
	print iterator, " frequencies out of ", total, " calculated"

print "All frequencies calculated"
print "Writing frequencies to file"
orfile = gzip.open("inverse_word_collection_frequencies.json.gz",'w')
orfile.write(json.dumps(frequencies)+"\n")
orfile.close()
print "Done... Exiting"


